from bs4 import BeautifulSoup
import time
import requests

url = 'https://knewone.com/discover?page='
headers = {
    'Cookie': '_ga=GA1.2.123069562.1516350198; _gid=GA1.2.1981471548.1516350198; Hm_lvt_b44696b80ba45a90a23982e53f8347d0=1516350198; gr_user_id=452be13a-8178-4d23-a287-f5b8d1e4f158; gr_session_id_e7b7e334c98d4530928513e7439f9ed2=7cfbd6a2-3b98-4a71-a80f-6133b8252d21; Hm_lpvt_b44696b80ba45a90a23982e53f8347d0=1516350261; _knewone_v2_session=SVh1V2dsUG9reksvejNDT04vU0VQQTJlWUpwdUQ4UE5xTHhnRzFFbzl4ZUxQWGFSQXRYc2ptYzFyQ1VjSk1pMS81cFlaejE5WGlnS2J6ZGYyUlllM3FRZlh4c0dSOWpWU2ZMUTR6UnZjTE9sWWsyeFV4Ny93SXIwOHBFSEdnSW5jcmdMN2MveHowOEl6cGl4cUFCelVlQ2R3OTVBQWs5N1hEYVoxZUI3V0wxazJWVUpHeUtDZE1OWFlpUWt2MVFXLS1ZNjd1c3htVXZYaXhIdUYrb2xFNjRRPT0%3D--bbaa7c355dad32e4cde089b3c53488f4466f5cee',
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}
link_url = 'https://knewone.com'

def get_wb_data(url, headers):
    wb_data = requests.get(url, headers=headers)
    soup = BeautifulSoup(wb_data.text, 'lxml')

    # 爬虫内容：标题，图片，详情链接
    titles = soup.select('section > h4 > a')
    imgs = soup.select('header > a > img')
    links = soup.select('article > header > a[target="_blank"]')

    for title, img, link in zip(titles, imgs, links):
        info = {
            'title': title.get_text(),
            'img': img.get('src'),
            'link': link_url + link.get('href')
        }
        print(info)


def get_page_data(start, end):
    for i in range(start, end):
        time.sleep(2)
        get_wb_data(url + str(i), headers=headers)


get_page_data(1, 10)
